Neural Style Transfer¶
Xinzhu Wang(xw2581), Ying Jin(yj2453), Jincheng Xu(jx2365), Xudong Guo(xg2305)
In [0]:
import os
import numpy as np
import scipy.io
import imageio
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from google.colab import files
from PIL import Image, ImageOps
import cv2
#import necessary packages
In [0]:
print("TF version: ",tf.__version__)
In [0]:
from google.colab import drive
drive.mount('/content/drive')
Preprocessing¶
In [0]:
def resize_image(image_path, width, height, save = True):
image = Image.open(image_path)
image = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
if save:
image_directory = image_path.split('/')
image_directory[-1] = 'resized_' + image_directory[-1]
output_path = '/'.join(image_directory)
if not os.path.exists(output_path):
image.save(output_path)
image = np.asarray(image, np.float32)
return np.expand_dims(image, 0)
#define resize image function
In [0]:
def noise_image(content_image, width, height, noise_ratio = 0.6):
noise_image = np.random.uniform(-20, 20, (1, height, width, 3)).astype(np.float32)
aggregated = noise_image * noise_ratio + content_image * (1 - noise_ratio)
return aggregated
#define noise image function
In [0]:
def preprocess(content_image_path,style_image_path):
content_image = resize_image(content_image_path, image_width, image_height)
style_image = resize_image(style_image_path, image_width, image_height)
noise_images = noise_image(content_image, image_width, image_height)
style_image -= MEAN_PIXEL
content_image -= MEAN_PIXEL
return style_image, content_image, noise_images
#define preprocess function
In [0]:
def inp(image_widths, image_heights):
with tf.variable_scope("input"):
input_img = tf.get_variable("in_img",
shape = ([1, image_heights, image_widths, 3]),
dtype = tf.float32,
initializer = tf.zeros_initializer())
return input_img
#define input_image function
Load VGG19 model¶
In [0]:
VGG19 = scipy.io.loadmat('drive/My Drive/5242_project/imagenet-vgg-verydeep-19.mat')
In [0]:
vgg_layer = VGG19['layers']
In [0]:
def weights(layer, expected_layer_name):
# Return the weights and bias from VGG for a given layer
weight = vgg_layer[0][layer][0][0][2][0][0] # (3,3,3,64)
bias = vgg_layer[0][layer][0][0][2][0][1] # (64,1)
layer_name = vgg_layer[0][layer][0][0][0][0] #current layer name
assert layer_name == expected_layer_name, print(expected_layer_name, layer_name)
return weight, bias
In [0]:
def conv_relu(previous_layer, current_layer, layer_name):
w, b = weights(current_layer, layer_name)
W = tf.constant(w)
B = tf.constant(np.reshape(b, b.size)) # 64
conv = tf.nn.conv2d(previous_layer, filters=W, strides=[1,1,1,1], padding='SAME')
return tf.nn.relu(conv + B)
In [0]:
def avg_pool(previous_layer):
return tf.nn.avg_pool(previous_layer, ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
In [0]:
layers = (
'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',
'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',
'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3',
'relu3_3', 'conv3_4', 'relu3_4', 'pool3',
'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',
'relu4_3', 'conv4_4', 'relu4_4', 'pool4',
'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3',
'relu5_3', 'conv5_4', 'relu5_4'
)
In [0]:
def vgg_model(inputs):
vgg = {}
layer_dict = {}
for layer_idx,layer_name in enumerate(layers):
layer_dict[layer_name]=layer_idx
vgg['input'] = inputs
vgg['conv1_1']=conv_relu(vgg['input'],layer_dict['conv1_1'],'conv1_1')
vgg['conv1_2']=conv_relu(vgg['conv1_1'],layer_dict['conv1_2'],'conv1_2')
vgg['avgpool1']=avg_pool(vgg['conv1_2'])
vgg['conv2_1']=conv_relu(vgg['avgpool1'],layer_dict['conv2_1'],'conv2_1')
vgg['conv2_2']=conv_relu(vgg['conv2_1'],layer_dict['conv2_2'],'conv2_2')
vgg['avgpool2']=avg_pool(vgg['conv2_2'])
vgg['conv3_1']=conv_relu(vgg['avgpool2'],layer_dict['conv3_1'],'conv3_1')
vgg['conv3_2']=conv_relu(vgg['conv3_1'],layer_dict['conv3_2'],'conv3_2')
vgg['conv3_3']=conv_relu(vgg['conv3_2'],layer_dict['conv3_3'],'conv3_3')
vgg['conv3_4']=conv_relu(vgg['conv3_3'],layer_dict['conv3_4'],'conv3_4')
vgg['avgpool3']=avg_pool(vgg['conv3_4'])
vgg['conv4_1']=conv_relu(vgg['avgpool3'],layer_dict['conv4_1'],'conv4_1')
vgg['conv4_2']=conv_relu(vgg['conv4_1'],layer_dict['conv4_2'],'conv4_2')
vgg['conv4_3']=conv_relu(vgg['conv4_2'],layer_dict['conv4_3'],'conv4_3')
vgg['conv4_4']=conv_relu(vgg['conv4_3'],layer_dict['conv4_4'],'conv4_4')
vgg['avgpool4']=avg_pool(vgg['conv4_4'])
vgg['conv5_1']=conv_relu(vgg['avgpool4'],layer_dict['conv5_1'],'conv5_1')
vgg['conv5_2']=conv_relu(vgg['conv5_1'],layer_dict['conv5_2'],'conv5_2')
vgg['conv5_3']=conv_relu(vgg['conv5_2'],layer_dict['conv5_3'],'conv5_3')
vgg['conv5_4']=conv_relu(vgg['conv5_3'],layer_dict['conv5_4'],'conv5_4')
vgg['avgpool5']=avg_pool(vgg['conv5_4'])
return vgg
#define vgg19 model
Loss Function¶
In [0]:
content_layer = "conv4_2"
style_layers = ["conv1_1", "conv2_1", "conv3_1", "conv4_1", "conv5_1"]
In [0]:
def _content_loss(P, F):
content_loss = tf.reduce_sum(tf.square(F - P)) / 2
return content_loss
In [0]:
def _gram_matrix(F, N, M):
F = tf.reshape(F, (M, N))
return tf.matmul(tf.transpose(F), F)
In [0]:
def _single_style_loss(a, g):
N = a.shape[3]
M = a.shape[1] * a.shape[2]
A = _gram_matrix(a, N, M)
G = _gram_matrix(g, N, M)
return tf.reduce_sum(tf.square(G - A)) / ((2 * N * M) ** 2)
In [0]:
def _style_loss(A):
n_layers = len(A)
E = [_single_style_loss(A[i], vgg[style_layers[i]]) for i in range(n_layers)]
style_loss = sum(style_layers_weights[i] * E[i] for i in range(n_layers))
return style_loss
In [0]:
def losses():
with tf.variable_scope("losses"):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(input_img.assign(content_image))
gen_img_content = vgg[content_layer]
content_img_content = sess.run(gen_img_content)
content_loss = _content_loss(content_img_content, gen_img_content)
with tf.Session() as sess:
sess.run(input_img.assign(style_image))
new_style_layers = sess.run([vgg[layer] for layer in style_layers])
style_loss = _style_loss(new_style_layers)
total_loss = content_loss_weight * content_loss + style_loss_weight * style_loss
return content_loss, style_loss, total_loss
# define the loss function
Model¶
In [0]:
def train(iteration=100, seed = 2019, show = False):
tf.set_random_seed(seed)
learning_rate = 2.0
optimizer = tf.train.AdamOptimizer(learning_rate)
training = optimizer.minimize(total_loss)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(input_img.assign(noise_images))
for epoch in range(iteration):
sess.run(training)
# generate a picture for every 20 epoches
if epoch == 0 or (epoch+1) % 20 == 0:
generated_image, total_losses = sess.run([input_img, total_loss])
generated_image = generated_image + MEAN_PIXEL
print('Step' + str(epoch + 1) )
print('Sum:' + str(np.sum(generated_image)))
print('Loss:'+ str(total_losses))
# save the picture
path = '%s/%d_outputs/%d_seed/%s_relative_weight' % (style_name,
iteration,
seed,
str(content_loss_weight))
if not os.path.exists(path):
os.makedirs(path)
filename = '%s/%d_outputs/%d_seed/%s_relative_weight/epoch_%d.png' % (style_name,
iteration,
seed,
str(content_loss_weight),
epoch+1)
generated_image = generated_image[0]
generated_image = np.clip(generated_image, 0, 255).astype('uint8')
imageio.imwrite(filename, generated_image)
if show == True:
plt.imshow(generated_image)
plt.axis('off')
# define the train function with seed 2019 and iteration 100 set by default
Test Result¶
In [0]:
content_path = 'drive/My Drive/5242_project/content5.jpg'
Image.open(content_path)
# input the content image
Out[0]:
In [0]:
style_path = 'drive/My Drive/5242_project/style2.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
Image.open(style_path)
Out[0]:
In [0]:
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
# build Style & Content Representations
In [0]:
content_loss_weight = 0.001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
# define the relevant weight and compute the loss
Results with Different Parameters¶
Iterations¶
Iteration = 100
In [0]:
train(iteration=100)
#training the model with 100 iterations
In [0]:
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
Iteration = 300
In [0]:
train(iteration=300)
#training the model with 300 iterations
In [0]:
path = 'style2/300_outputs/2019_seed/0.001_relative_weight/epoch_300.png'
Image.open(path)
Out[0]:
Iteration = 500
In [0]:
train(iteration=500)
#training the model with 500 iterations
In [0]:
path = 'style2/500_outputs/2019_seed/0.001_relative_weight/epoch_500.png'
Image.open(path)
Out[0]:
Iteration = 1000
In [0]:
train(iteration=1000)
#training the model with 1000 iterations
In [0]:
path = 'style2/1000_outputs/2019_seed/0.001_relative_weight/epoch_1000.png'
Image.open(path)
Out[0]:
Random Seeds¶
seed = 0
In [0]:
train(seed=0)
#training the model with seed 0
In [0]:
path = 'style2/100_outputs/0_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
seed = 2019
In [0]:
train(seed=2019)
#training the model with seed 2019
In [0]:
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
seed = 5242
In [0]:
train(seed = 5242)
#training the model with seed 5242
In [0]:
path = 'style2/100_outputs/5242_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
Relative weights¶
relative_weight = 0.01
In [0]:
tf.reset_default_graph()
# content_path = 'drive/My Drive/42 Project/content.jpg'
# style_path = 'drive/My Drive/42 Project/style.jpg'
# content_path = 'drive/My Drive/content.jpg'
# style_path = 'drive/My Drive/style.jpg'
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.01
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train()
#training the model with 0.01, 0.001 relative weight
In [0]:
path = 'style2/100_outputs/2019_seed/0.01_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
relative_weight = 0.001
In [0]:
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
relative_weight = 0.0001
In [0]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train()
#training the model with 0.0001 relative weight
In [0]:
path = 'style2/100_outputs/2019_seed/0.0001_relative_weight/epoch_100.png'
Image.open(path)
Out[0]:
After parameter tuning process, we decide to use our model with 1000 iterations, 0.0001 relative weights and 2019 (default) seed.
Final Model Training¶
Overlay Van Gogh on temple¶
In [32]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
In [7]:
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[7]:
Overlay Van Gogh on cafe¶
In [0]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content2.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
In [8]:
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[8]:
Overlay Van Gogh on One Piece¶
In [0]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content10.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
In [9]:
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[9]:
Overlay Picasso on temple¶
In [26]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
In [12]:
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[12]:
Overlay Picasso on cafe¶
In [28]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content2.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
In [11]:
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[11]:
Overlay Picasso on One Piece¶
In [30]:
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content10.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
In [10]:
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
Out[10]:
